#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@创建时间    : 2022/11/18  16:02
@作者  : st
@文件名: data_process_medical_1118.py
@项目名: PyCharm
@文件描述:
    
"""
import json
import math
import os

import pandas as pd

from utils.constent import data_path

data_dir = os.path.join(data_path, 'matadata', 'medical_1118')


def get_medical_tag_datas():
    all_text_list = []
    all_res_list = []
    all_label_list = []
    file_list = os.listdir(data_dir)
    for f in file_list:
        fp = os.path.join(data_dir, f)
        text_list, res_list, label_dict = export_data_train(fp)
        all_text_list.extend(text_list)
        all_res_list.extend(res_list)
        all_label_list.extend(label_dict)
    return all_text_list, all_res_list, all_label_list


def export_data_train(matadata_export):
    """
    标注导出csv文件转换为训练文件
    Returns:
    """
    df = pd.read_csv(matadata_export)
    data_len = df['text'].count()
    print(data_len)
    text_list = []
    res_list = []
    label_list = []
    for i in range(data_len):
        text = df['text'][i].replace('\t\t\t', '\t').replace('\t', '')
        label = df['label'][i]
        if isinstance(label, float) and math.isnan(label):
            continue
        label = json.loads(label)
        tag_word_list, labels = text_screening(text, label)
        text_list.append(text)
        res_list.append(tag_word_list)
        label_list.append(labels)
    return text_list, res_list, label_list


def text_screening(text, label):
    """
    文本过滤
    :param text:
    :param label:
    :return:
    """
    # 文本过滤
    tag_word_list = list()
    # 判断标签是否存在于文本中
    labels = dict()
    for item in label:
        temp_word = item['text'].replace('\t\t\t', '').replace('\t', '').replace(' ', '')
        if not (item.__contains__('text') and text.__contains__(temp_word)):
            continue
        tag_word_list.append(temp_word)
        if not labels.__contains__(temp_word):
            labels[temp_word] = []
        labels[temp_word].append({'text': temp_word, 'start': item['start'], 'end': item['end']})
    return tag_word_list, labels


if __name__ == '__main__':
    all_text_list, all_res_list = get_medical_tag_datas()
    print('-')